from __future__ import division
from operator import itemgetter, attrgetter

import gc
import math
import matplotlib
import os
import pylab
import random
import sys
import time
from sets import Set

def generate_final_partialBM25_score_distribution_result():
    
    # key: class label in int format
    # value: currentLowerBound in float format
    classLabelsWithTheirLowerBoundDict = {}
    
    # key: class label in int format
    # value: # of ALL related postings belonging to this class
    classLabelsWithNumOfALLRelatedPostingsDict = {}
    
    # key: class label in int format
    # value: # of TOP10 related postings belonging to this class
    classLabelsWithNumOfTOP10RelatedPostingsDict = {}
    
    sMin = 0.001
    sMax = 19.746
    stepFactor = 1.2
    
    currentLowerBound = sMin
    classLabel = 0
    while currentLowerBound <= 25:
        # print classLabel,currentLowerBound
        if classLabel not in classLabelsWithTheirLowerBoundDict:
            classLabelsWithTheirLowerBoundDict[classLabel] = currentLowerBound
            classLabelsWithNumOfALLRelatedPostingsDict[classLabel] = 0
            classLabelsWithNumOfTOP10RelatedPostingsDict[classLabel] = 0
        currentLowerBound = currentLowerBound * 1.2
        classLabel += 1
    
    print "len(classLabelsWithTheirLowerBoundDict):",len(classLabelsWithTheirLowerBoundDict)
    
    # CURRENT version pending since 2013/08/27
    inputFileName0 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/ALLRelatedPostingsBucketingIntoClassesFromHead95KQueries"
    # OLD NOT correct version
    # inputFileName0 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/ALLRelatedPostingsBucketingIntoClassesFromHead95KQueries"
    inputFileHandler = open(inputFileName0,"r")
    for line in inputFileHandler.readlines():
        lineElements = line.strip().split(" ")
        currentOriginalRoundingClassInFloatFormat = float(lineElements[0])
        numOfPostingsInCurrentRoundingClass = int( lineElements[1] )
        
        if numOfPostingsInCurrentRoundingClass != 0:
            beenCountedFlag = False
            for classLabel in classLabelsWithTheirLowerBoundDict:
                # for DEBUG
                # print "currentOriginalRoundingClassInFloatFormat:",currentOriginalRoundingClassInFloatFormat
                # print "classLabelsWithTheirLowerBoundDict[classLabel]:",classLabelsWithTheirLowerBoundDict[classLabel]
                if classLabel == 0:
                    pass
                    # for DEBUG
                    # print "classLabelsWithTheirLowerBoundDict[classLabel-1]:",classLabelsWithTheirLowerBoundDict[classLabel-1]
                elif classLabel != 0:
                    if currentOriginalRoundingClassInFloatFormat < classLabelsWithTheirLowerBoundDict[classLabel] and currentOriginalRoundingClassInFloatFormat >= classLabelsWithTheirLowerBoundDict[classLabel-1]:
                        classLabelsWithNumOfALLRelatedPostingsDict[classLabel-1] += numOfPostingsInCurrentRoundingClass
                        beenCountedFlag = True
                    else:
                        pass
                else:
                    pass # just don't care
            if beenCountedFlag:
                pass
            else:
                print "currentOriginalRoundingClassInFloatFormat:",currentOriginalRoundingClassInFloatFormat
                print "numOfPostingsInCurrentRoundingClass:",numOfPostingsInCurrentRoundingClass
                exit(1)
    inputFileHandler.close()
    
    # CURRENT version and I think it is OK
    inputFileName1 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/TOP10RelatedPostingsBucketingIntoClassesFromHead95KQueries"
    inputFileHandler = open(inputFileName1,"r")
    for line in inputFileHandler.readlines():
        lineElements = line.strip().split(" ")
        currentOriginalRoundingClassInFloatFormat = float(lineElements[0])
        numOfPostingsInCurrentRoundingClass = int( lineElements[1] )
        if numOfPostingsInCurrentRoundingClass != 0:
            for classLabel in classLabelsWithTheirLowerBoundDict:
                # for DEBUG
                # print "currentOriginalRoundingClassInFloatFormat:",currentOriginalRoundingClassInFloatFormat
                # print "classLabelsWithTheirLowerBoundDict[classLabel]:",classLabelsWithTheirLowerBoundDict[classLabel]
                if classLabel != 0:
                    pass
                    # for DEBUG
                    # print "classLabelsWithTheirLowerBoundDict[classLabel-1]:",classLabelsWithTheirLowerBoundDict[classLabel-1]
                
                if classLabel != 0:
                    if currentOriginalRoundingClassInFloatFormat < classLabelsWithTheirLowerBoundDict[classLabel] and currentOriginalRoundingClassInFloatFormat >= classLabelsWithTheirLowerBoundDict[classLabel-1]:
                        classLabelsWithNumOfTOP10RelatedPostingsDict[classLabel-1] += numOfPostingsInCurrentRoundingClass
                    else:
                        pass
                else:
                    pass # just don't care
    inputFileHandler.close()
    
    print "len(classLabelsWithTheirLowerBoundDict):",len(classLabelsWithTheirLowerBoundDict)
    print "len(classLabelsWithNumOfALLRelatedPostingsDict):",len(classLabelsWithNumOfALLRelatedPostingsDict)
    print "len(classLabelsWithNumOfTOP10RelatedPostingsDict):",len(classLabelsWithNumOfTOP10RelatedPostingsDict)
    assert len(classLabelsWithTheirLowerBoundDict) == len(classLabelsWithNumOfALLRelatedPostingsDict)
    
    for i in range( 0,len(classLabelsWithTheirLowerBoundDict) ):
        if classLabelsWithNumOfALLRelatedPostingsDict[i] != 0:
            print i,classLabelsWithTheirLowerBoundDict[i],classLabelsWithNumOfALLRelatedPostingsDict[i],classLabelsWithNumOfTOP10RelatedPostingsDict[i],classLabelsWithNumOfTOP10RelatedPostingsDict[i]/classLabelsWithNumOfALLRelatedPostingsDict[i]
        else:
            print i,classLabelsWithTheirLowerBoundDict[i],classLabelsWithNumOfALLRelatedPostingsDict[i],classLabelsWithNumOfTOP10RelatedPostingsDict[i],0
    
    # check
    # total Num Of ALL Related Postings Counted: ?
    # total Num Of TOP10 Related Postings Counted: 3632263
    tempSumValue1 = 0
    tempSumValue2 = 0
    for i in range( 0,len(classLabelsWithTheirLowerBoundDict) ):
        tempSumValue1 += classLabelsWithNumOfALLRelatedPostingsDict[i]
        tempSumValue2 += classLabelsWithNumOfTOP10RelatedPostingsDict[i]
    print "ALLRelatedPostings:",tempSumValue1
    print "TOP10RelatedPostings:",tempSumValue2
    print "Probability for a random posting to be in the TOP10:",tempSumValue2/tempSumValue1
    print "sMin:",sMin
    print "sMax:",sMax
    print "stepFactor:",stepFactor
    # Should have new numbers added here
    assert tempSumValue1 == 1340249690161
    assert tempSumValue2 == 3632263


def stepX_find_Smin_AND_Smax():
    print "stepX_find_Smin_AND_Smax called."
    # inputFileName1 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/TOP10RelatedPostingsBucketingIntoClassesFromHead95KQueries"
    inputFileName2 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/ALLRelatedPostingsBucketingIntoClassesFromHead95KQueries"
    inputFileHandler = open(inputFileName2,"r")
    
    minClassLabelValueWhichHasElements = 99.999
    maxClassLabelValueWhichHasElements = 0.0
    
    numOfPostingsBelongToTheMinClassLabel = 0
    numOfPostingsBelongToTheMaxClassLabel = 0
    
    for line in inputFileHandler.readlines():
        lineElements = line.strip().split(" ")
        roundingClassLabelValue = float( lineElements[0] )
        numOfPostingsBelongToThisClass = int( lineElements[1] )
        if roundingClassLabelValue > maxClassLabelValueWhichHasElements and numOfPostingsBelongToThisClass != 0:
            maxClassLabelValueWhichHasElements = roundingClassLabelValue
            numOfPostingsBelongToTheMaxClassLabel = numOfPostingsBelongToThisClass
        
        if roundingClassLabelValue < minClassLabelValueWhichHasElements and numOfPostingsBelongToThisClass != 0:
            minClassLabelValueWhichHasElements = roundingClassLabelValue
            numOfPostingsBelongToTheMinClassLabel = numOfPostingsBelongToThisClass
            
    print "maxClassLabelValueWhichHasElements:",maxClassLabelValueWhichHasElements
    print "numOfPostingsBelongToTheMaxClassLabel:",numOfPostingsBelongToTheMaxClassLabel
    
    print "minClassLabelValueWhichHasElements:",minClassLabelValueWhichHasElements
    print "numOfPostingsBelongToTheMinClassLabel:",numOfPostingsBelongToTheMinClassLabel
    
    inputFileHandler.close()
    
    
    
    


def getQueryIDAndResultSetGivenResultFile(inputResultFileName,NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE,insideQueryIDAndResultSetDict,insideQueryIDAndQueryTermsDictDict,classLowerBoundAndTOP10PostingsCounterDict):
    print "getQueryIDAndResultSetGivenResultFile() function called."
    numOfQueriesHavingQID = 0
    numOfQueriesHavingSearchContent = 0
    numOfQueriesHavingSearchResults = 0
    
    inputFileHandler1 = open(inputResultFileName,"r")
    
    
    
    currentLine = inputFileHandler1.readline()
    nextLine = ""
    currentQID = ""
    
    numOfResultsForTheAllQueries = 0
    
    while currentLine:
        # sampling parsing line:
        # qid: 701
        if currentLine.startswith("qid:"):
            currentQID = int( currentLine.strip().split(":")[1].strip() )
            print "currentQID:",str( currentQID )
            # init the variable
            if currentQID not in insideQueryIDAndResultSetDict:
                insideQueryIDAndResultSetDict[currentQID] = []
            numOfQueriesHavingQID += 1
        
        # sample parsing line:
        # Search: u s  oil industry history
        if currentLine.startswith("Search:"):
            elementList = currentLine.strip().split(" ")
            
            currentSearchContent = ""
            
            queryTermList = []
            for element in elementList[1:]:
                if element.strip() != "":
                    queryTermList.append( element.strip() )        
            currentQueryLength = len(queryTermList)
            print "currentQueryLength:",currentQueryLength
    
            for element in queryTermList:
                currentSearchContent += element + " "
            
            print currentSearchContent
            numOfQueriesHavingSearchContent += 1
            
            
            currentQueryTermIndexDict = {}
            nextLine = inputFileHandler1.readline()
            #print nextLine.strip()
            if nextLine.strip() != "":
                # I think I can skip the line: 
                # [2013-05-24 14:20:44] 38871 terms with their lists length have been loaded.
                nextLine = inputFileHandler1.readline()
                nextLineElements = nextLine.strip().split(" ")
                # print "nextLineElements:",nextLineElements
                # sample parsing line: 
                # oil:0 industry:1 history:2 u:3 s:4
                for element in nextLineElements:
                    if element.split(":")[0] not in currentQueryTermIndexDict:
                        currentQueryTermIndexDict[ int(element.split(":")[1]) ] = element.split(":")[0]
                
                print "currentQueryTermIndexDict:",currentQueryTermIndexDict
                # print "len(currentQueryTermIndexDict):",len(currentQueryTermIndexDict)
                if currentQID not in insideQueryIDAndQueryTermsDictDict:
                    insideQueryIDAndQueryTermsDictDict[currentQID] = currentQueryTermIndexDict

            
                
                # Now, it is time to read the threshold line
                currentLine = inputFileHandler1.readline()
                if len(currentLine.strip().split(":")) == 2:
                    name = currentLine.strip().split(":")[0]
                    value = currentLine.strip().split(":")[1]
                    print name,"::",value
                else:
                    print "format problem"
                    exit(1)
                    
                # Now, it is time to read computation method line
                currentLine = inputFileHandler1.readline()
                if len(currentLine.strip().split(":")) == 2:
                    name = currentLine.strip().split(":")[0]
                    value = currentLine.strip().split(":")[1]
                    print name,"::",value
                else:
                    print "format problem"
                    exit(1)
                
                # skip the line like the following:
                # 3 factor probability formula logic implemented by Wei on 2013/08/02 at school    
                # currentLine = inputFileHandler1.readline()
                
                # Now, it is time to read the actual training example line
                currentLine = inputFileHandler1.readline()
                numOfResultsForTheCurrentQuery = 0
                currentResultLines = []
                # for DEBUG ONLY(IMPOERTANT debug setting here)
                # print "len( currentLine.strip().split(" ") ):",len( currentLine.strip().split(" ") )
                # print "currentLine:",currentLine
                # exit(1)
                lineElements = currentLine.strip().split(" ")
                while len( lineElements ) == 35:
                    theRank = int(lineElements[0])
                    theDocID = lineElements[-2]
                    theTrecID = lineElements[-1]

                    if theRank <= NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE:
                        insideQueryIDAndResultSetDict[currentQID].append( (theTrecID,theDocID) )
                        # pick up the BM25 scores if the rank <= NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE
                        base = 1
                        LENGTH = -1
                        if len(currentQueryTermIndexDict) <= 10:
                            LENGTH = len(currentQueryTermIndexDict)
                        else:
                            LENGTH = 10
                        
                        for i in range(0,LENGTH):
                            ###########################
                            # get the posting score and ready to bucket them
                            postingScoreInStringFormat = lineElements[base + i]
                            postingScoreInFloatFormat = float(postingScoreInStringFormat)
                            postingScoreInStringFormat = str(round(postingScoreInFloatFormat,3))
                            # debug
                            # print "postingScoreInFloatFormat:",postingScoreInFloatFormat
                            # print "postingScoreInStringFormat:",postingScoreInStringFormat
                            # print 
                            if postingScoreInStringFormat in classLowerBoundAndTOP10PostingsCounterDict:
                                classLowerBoundAndTOP10PostingsCounterDict[postingScoreInStringFormat] += 1
                            else:
                                print "shit, there is no cell to fit."
                                print "postingScoreInStringFormat:",postingScoreInStringFormat
                                exit(1)
                            ###########################
                    
                    numOfResultsForTheCurrentQuery += 1
                    currentResultLines.append(currentLine.strip())
                    currentLine = inputFileHandler1.readline()
                    lineElements = currentLine.strip().split(" ")
                   
                numOfResultsForTheAllQueries += numOfResultsForTheCurrentQuery
                
                # sampling method 1: among all the top10 documents, select those documents, open those documents and extract the corresponding postings.
                # This method make sure that once the document have been selected, ALL the coresponding postings will also be selected.
                # This sampling method has been developed on 2013/01/09
                
                print "numOfResultsForTheCurrentQuery:",numOfResultsForTheCurrentQuery
                print "len(currentResultLines):",len(currentResultLines)
                print
            
        currentLine = inputFileHandler1.readline()

def computeTOP10PostingsPartialBM25ScoreDistribution():
    # define 100000 classes for the TOP10 partialBM25 score
    # key: class lower bound in string format
    # value: counter for this class
    currentLowerBoundInFloatFormat = 0.0
    step = 0.001
    classLowerBoundAndTOP10PostingsCounterDict = {}
    currentLowerBoundInStringFormat = str(currentLowerBoundInFloatFormat)
    
    
    for i in range(0,100000):
        if currentLowerBoundInStringFormat not in classLowerBoundAndTOP10PostingsCounterDict:
            # print "currentLowerBoundInStringFormat:",currentLowerBoundInStringFormat
            classLowerBoundAndTOP10PostingsCounterDict[currentLowerBoundInStringFormat] = 0
        else:
            print "duplicated lower bounds."
            exit(1)
        currentLowerBoundInFloatFormat += step
        currentLowerBoundInStringFormat = str(currentLowerBoundInFloatFormat)
    
    print "len(classLowerBoundAndTOP10PostingsCounterDict):",len(classLowerBoundAndTOP10PostingsCounterDict)
    
    
    # for production
    inputResultFileName = "/data1/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/raw_TOP1000_results_for_head_95K_queries_output_from_IRTK"
    # for debug
    # inputResultFileName = "/data1/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/raw_TOP1000_results_for_head_95K_queries_output_from_IRTK_head_2_queries"               
    
    TOPKValueTakenIntoConsideration = 10
    
    # sub_step1: load the results from the raw result file
    # key: queryID 
    # value: a list of tuples with the format (rank,trecID,docID)
    outsideQueryIDAndResultSetDict = {}
    # key: queryID
    # value: the query term dict
    outsideQueryIDAndQueryTermsDictDict = {}
    
    # 3 arguments:
    # (1) inputResultFileName
    # (3) NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE
    getQueryIDAndResultSetGivenResultFile(inputResultFileName,TOPKValueTakenIntoConsideration,outsideQueryIDAndResultSetDict,outsideQueryIDAndQueryTermsDictDict,classLowerBoundAndTOP10PostingsCounterDict)
    
    print "len(outsideQueryIDAndResultSetDict):",len(outsideQueryIDAndResultSetDict)
    print "len(outsideQueryIDAndQueryTermsDictDict):",len(outsideQueryIDAndQueryTermsDictDict)

    totalNumOfPostingsCounted = 0
    for classLowerBound in classLowerBoundAndTOP10PostingsCounterDict:
        if classLowerBoundAndTOP10PostingsCounterDict[classLowerBound] != 0:
            # print classLowerBound,classLowerBoundAndTOP10PostingsCounterDict[classLowerBound]
            totalNumOfPostingsCounted += classLowerBoundAndTOP10PostingsCounterDict[classLowerBound]
        else:
            pass
    print "totalNumOfPostingsCounted:",totalNumOfPostingsCounted
    
    currentLowerBoundInFloatFormat = 0.0
    step = 0.001
    currentLowerBoundInStringFormat = str(currentLowerBoundInFloatFormat)
    
    for i in range(0,100000):
        print currentLowerBoundInStringFormat,classLowerBoundAndTOP10PostingsCounterDict[currentLowerBoundInStringFormat]
        currentLowerBoundInFloatFormat += step
        currentLowerBoundInStringFormat = str(currentLowerBoundInFloatFormat)



def computeWHOLEPostingsPartialBM25ScoreDistribution():
    print "------>Times the real freq of a term in the head95K query log INCLUDED."
    # for DEBUG
    # This is just the test part
    # print round(0.001,3)
    # exit(1)
    
    
    # step1: get the set of query terms from 95K queries which need to analyze
    # # of query terms included in the head95K: 37817
    queryTermHead95KQueriesDict = {}
    # for Production
    inputFileName1 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/realFreqOfTermsIn_100KQueries_0_1_95%"
    # for DEBUG
    # inputFileName1 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/realFreqOfTermsIn_100KQueries_0_1_95%_DEBUG"
    inputFileHanlder = open(inputFileName1,"r")
    for line in inputFileHanlder.readlines():
        lineElements = line.strip().split(" ")
        currentTerm = lineElements[0]
        currentTermRealFreqIn95KQueires = int(lineElements[1])
        # ignore the lineElements[1] cause I don't need this info
        if currentTerm not in queryTermHead95KQueriesDict:
            queryTermHead95KQueriesDict[currentTerm] = currentTermRealFreqIn95KQueires
        else:
            print "duplicated terms found."
            exit(1)
    print "len(queryTermHead95KQueriesDict):",len(queryTermHead95KQueriesDict)
    print "queryTermHead95KQueriesDict['of']:",queryTermHead95KQueriesDict['of']
    print "queryTermHead95KQueriesDict['fawn']:",queryTermHead95KQueriesDict['fawn']
    inputFileHanlder.close()
    
    # step2: define 100000 classes for the partialBM25 score
    # key: class lower bound in string format
    # value: counter for this class
    currentClassValueInFloatFormat = 0.0
    step = 0.001
    classValueAndItsCounterDict = {}
    currentLowerBoundInStringFormat = str(currentClassValueInFloatFormat)
    
    
    for i in range(0,100000):
        if currentLowerBoundInStringFormat not in classValueAndItsCounterDict:
            # print "currentLowerBoundInStringFormat:",currentLowerBoundInStringFormat
            classValueAndItsCounterDict[currentLowerBoundInStringFormat] = 0
        else:
            print "duplicated lower bounds."
            exit(1)
        currentClassValueInFloatFormat += step
        currentLowerBoundInStringFormat = str(currentClassValueInFloatFormat)
    
    print "len(classValueAndItsCounterDict):",len(classValueAndItsCounterDict)
    
    
    # step3: put the partialBM25 scores into the 1000 classes
    # key: query term in string format
    # value: a list contains 4 items as following:
        # termIndexNumber
        # invertedIndexLength
        # invertedIndexBeginningPosition
        # invertedIndexEndingPosition
    queryTermInvertedIndexInfo = {}
    
    inputDataSourceFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/rawTermInvertedIndexOutput_with_trecID_added.txt"
    inputDataSourceFileHandler = open(inputDataSourceFileName,"r")
    
    inputAuxSourceFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/rawTermInvertedIndexOutputAccessAuxFile_NEW_20121128.txt"
    inputAuxSourceFileHandler = open(inputAuxSourceFileName,"r")
    
    for line in inputAuxSourceFileHandler.readlines():
        lineElements = line.strip().split(" ")
        
        termIndexNumber = int(lineElements[0])
        term = lineElements[1]
        invertedIndexLength = int(lineElements[2])
        invertedIndexBeginningPosition = int(lineElements[3])
        invertedIndexEndingPosition = int(lineElements[4])
        
        if term not in queryTermInvertedIndexInfo:
            queryTermInvertedIndexInfo[term] = []
            # index 0
            queryTermInvertedIndexInfo[term].append(termIndexNumber)
            # index 1
            queryTermInvertedIndexInfo[term].append(invertedIndexLength)
            # index 2
            queryTermInvertedIndexInfo[term].append(invertedIndexBeginningPosition)
            # index 3
            queryTermInvertedIndexInfo[term].append(invertedIndexEndingPosition)   
        else:
            print "unnormal, mark1"
            exit(1)
    
    print "len(queryTermInvertedIndexInfo):",len(queryTermInvertedIndexInfo)
    
    
    print "Check Begins..."
    for term in queryTermHead95KQueriesDict:
        if term in queryTermInvertedIndexInfo:
            pass
        else:
            print "term",term,"NOT found."
            exit(1)
    print "len(queryTermHead95KQueriesDict):",len(queryTermHead95KQueriesDict)
    print "len(queryTermInvertedIndexInfo):",len(queryTermInvertedIndexInfo)
    print "Check Ends."

    
    for index,term in enumerate(queryTermHead95KQueriesDict):
        currentTermInvertedIndexCorrectLength = queryTermInvertedIndexInfo[term][1]
        currentTermInvertedIndexBeginningPosition = queryTermInvertedIndexInfo[term][2]
        currentTermInvertedIndexEndingPosition = queryTermInvertedIndexInfo[term][3]
        
        print "------>term:",term
        print "------>real freq of term in head 95K queries:",queryTermHead95KQueriesDict[term]
        print "------>currentTermInvertedIndexCorrectLength:",currentTermInvertedIndexCorrectLength
        # print "------>currentTermInvertedIndexBeginningPosition:",currentTermInvertedIndexBeginningPosition
        # print "------>currentTermInvertedIndexEndingPosition:",currentTermInvertedIndexEndingPosition
        
        if currentTermInvertedIndexCorrectLength != 0:
            inputDataSourceFileHandler.seek(currentTermInvertedIndexBeginningPosition)
            
            currentDataLine = inputDataSourceFileHandler.readline()
            # print "first line:",currentDataLine
            
            while inputDataSourceFileHandler.tell() <= currentTermInvertedIndexEndingPosition:
                
                currentDataLineElements = currentDataLine.strip().split(" ")
                
                # get the posting score and ready to bucket them
                postingScoreInStringFormat = currentDataLineElements[2]
                postingScoreInFloatFormat = float(postingScoreInStringFormat)
                postingScoreInStringFormat = str(round(postingScoreInFloatFormat,3))
                # debug
                # print "postingScoreInFloatFormat:",postingScoreInFloatFormat
                # print "postingScoreInStringFormat:",postingScoreInStringFormat
                # print
                
                # CURRENT version is increasing by the real freq of the query term in head 95K queries
                if postingScoreInStringFormat in classValueAndItsCounterDict:
                    classValueAndItsCounterDict[postingScoreInStringFormat] += queryTermHead95KQueriesDict[term]
                else:
                    print "shit, there is no cell to fit."
                    print "postingScoreInStringFormat:",postingScoreInStringFormat
                    exit(1)                
                
                # OLD version is increasing by ONE 
                #if postingScoreInStringFormat in classValueAndItsCounterDict:
                #    classValueAndItsCounterDict[postingScoreInStringFormat] += 1
                #else:
                #    print "shit, there is no cell to fit."
                #    print "postingScoreInStringFormat:",postingScoreInStringFormat
                #    exit(1) 
                
                currentDataLine = inputDataSourceFileHandler.readline()
            
            # dealing with the last line
            # print "last line:",currentDataLine
            currentDataLineElements = currentDataLine.strip().split(" ")
            
            # get the posting score and ready to bucket them
            postingScoreInStringFormat = currentDataLineElements[2]
            postingScoreInFloatFormat = float(postingScoreInStringFormat)
            postingScoreInStringFormat = str(round(postingScoreInFloatFormat,3))
            # debug
            # print "postingScoreInFloatFormat:",postingScoreInFloatFormat
            # print "postingScoreInStringFormat:",postingScoreInStringFormat
            # print 

            # CURRENT version is increasing by the real freq of the query term in head 95K queries
            if postingScoreInStringFormat in classValueAndItsCounterDict:
                classValueAndItsCounterDict[postingScoreInStringFormat] += queryTermHead95KQueriesDict[term]
            else:
                print "shit, there is no cell to fit."
                print "term:",term
                print "postingScoreInStringFormat:",postingScoreInStringFormat
                exit(1)
                        
            # OLD version is increasing by ONE
            #if postingScoreInStringFormat in classValueAndItsCounterDict:
            #    classValueAndItsCounterDict[postingScoreInStringFormat] += 1
            #else:
            #    print "shit, there is no cell to fit."
            #    print "term:",term
            #    print "postingScoreInStringFormat:",postingScoreInStringFormat
            #    exit(1)
    
        else:
            print "query term:",term,"is NOT in the lexicon."
        # for DEBUG break
        #if index == 3:
        #    break
        
    totalNumOfPostingsCounted = 0
    for classLowerBound in classValueAndItsCounterDict:
        if classValueAndItsCounterDict[classLowerBound] != 0:
            # print classLowerBound,classValueAndItsCounterDict[classLowerBound]
            totalNumOfPostingsCounted += classValueAndItsCounterDict[classLowerBound]
        else:
            pass
    print "totalNumOfPostingsCounted:",totalNumOfPostingsCounted
    
    # step4: output the info from the memory to the disk
    currentClassValueInFloatFormat = 0.0
    step = 0.001
    currentLowerBoundInStringFormat = str(currentClassValueInFloatFormat)
    
    # for PRODUCTION
    for i in range(0,100000):
        print currentLowerBoundInStringFormat,classValueAndItsCounterDict[currentLowerBoundInStringFormat]
        currentClassValueInFloatFormat += step
        currentLowerBoundInStringFormat = str(currentClassValueInFloatFormat)
    
    # for DEBUG
    #for i in range(0,100000):
    #    if classValueAndItsCounterDict[currentLowerBoundInStringFormat] != 0:
    #        print currentLowerBoundInStringFormat,classValueAndItsCounterDict[currentLowerBoundInStringFormat]
    #    currentClassValueInFloatFormat += step
    #    currentLowerBoundInStringFormat = str(currentClassValueInFloatFormat)
           
print "Program Begins..."
# Updated by Wei 2013/08/27 night
# step1:
# This will take some time cause there are lots of info read from the disk
# computeWHOLEPostingsPartialBM25ScoreDistribution()

# Updated by Wei 2013/08/27 night
# step2:
# computeTOP10PostingsPartialBM25ScoreDistribution()

# Updated by Wei 2013/08/27 night
# step3:
# stepX_find_Smin_AND_Smax()

# Updated by Wei 2013/08/28 morning
# step4:
generate_final_partialBM25_score_distribution_result()

print "Program Ends."



